{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "using ReinforcementLearningAnIntroduction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "using Plots, StatsBase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "mutable struct CountOfLeft <: AbstractHook\n",
    "    counts::Vector{Int}\n",
    "    count::Int\n",
    "    CountOfLeft() = new([],0)\n",
    "end\n",
    "\n",
    "function (f::CountOfLeft)(::PreActStage, agent, env, obs, action)\n",
    "    if get_state(obs) == 1 && action == MaximizationBias.LEFT\n",
    "        f.count += 1\n",
    "    end\n",
    "end\n",
    "\n",
    "function (f::CountOfLeft)(::PostEpisodeStage, agent, env, obs)\n",
    "    push!(f.counts, f.count)\n",
    "    f.count = 0\n",
    "end\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3, 10)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env = MaximizationBiasEnv()\n",
    "ns, na = length(get_observation_space(env)), length(get_action_space(env))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "init_Q_agent (generic function with 1 method)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "init_double_Q_agent() = Agent(\n",
    "    policy=QBasedPolicy(\n",
    "        learner=DoubleLearner(\n",
    "            L1=TDLearner(\n",
    "                approximator=TabularApproximator(n_state=ns, n_action=na),\n",
    "                optimizer=Descent(0.1),\n",
    "                method=:SARS\n",
    "                ),\n",
    "            L2=TDLearner(\n",
    "                approximator=TabularApproximator(n_state=ns, n_action=na),\n",
    "                optimizer=Descent(0.1),\n",
    "                method=:SARS\n",
    "                )\n",
    "            ),\n",
    "        explorer=EpsilonGreedyExplorer(0.1)\n",
    "        ),\n",
    "    trajectory=EpisodicCompactSARTSATrajectory()\n",
    ")\n",
    "\n",
    "init_Q_agent() = Agent(\n",
    "    policy=QBasedPolicy(\n",
    "        learner=TDLearner(\n",
    "            approximator=TabularApproximator(n_state=ns, n_action=na),\n",
    "            optimizer=Descent(0.1),\n",
    "            method=:SARS\n",
    "            ),\n",
    "        explorer=EpsilonGreedyExplorer(0.1)\n",
    "        ),\n",
    "    trajectory=EpisodicCompactSARTSATrajectory()\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32mProgress: 100%|█████████████████████████████████████████| Time: 0:00:00\u001b[39m\n"
     ]
    },
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" width=\"600\" height=\"400\" viewBox=\"0 0 2400 1600\">\n",
       "<defs>\n",
       "  <clipPath id=\"clip4000\">\n",
       "    <rect x=\"0\" y=\"0\" width=\"2400\" height=\"1600\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<path clip-path=\"url(#clip4000)\" d=\"\n",
       "M0 1600 L2400 1600 L2400 0 L0 0  Z\n",
       "  \" fill=\"#ffffff\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<defs>\n",
       "  <clipPath id=\"clip4001\">\n",
       "    <rect x=\"480\" y=\"0\" width=\"1681\" height=\"1600\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<path clip-path=\"url(#clip4000)\" d=\"\n",
       "M153.898 1487.47 L2352.76 1487.47 L2352.76 47.2441 L153.898 47.2441  Z\n",
       "  \" fill=\"#ffffff\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<defs>\n",
       "  <clipPath id=\"clip4002\">\n",
       "    <rect x=\"153\" y=\"47\" width=\"2200\" height=\"1441\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<polyline clip-path=\"url(#clip4002)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  209.192,1487.47 209.192,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4002)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  902.969,1487.47 902.969,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4002)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  1596.75,1487.47 1596.75,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4002)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  2290.52,1487.47 2290.52,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4002)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  153.898,1470.24 2352.76,1470.24 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4002)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  153.898,1166.68 2352.76,1166.68 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4002)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  153.898,863.131 2352.76,863.131 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4002)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  153.898,559.577 2352.76,559.577 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4002)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  153.898,256.023 2352.76,256.023 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4000)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  153.898,1487.47 2352.76,1487.47 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4000)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  153.898,1487.47 153.898,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4000)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  209.192,1487.47 209.192,1470.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4000)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  902.969,1487.47 902.969,1470.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4000)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1596.75,1487.47 1596.75,1470.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4000)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  2290.52,1487.47 2290.52,1470.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4000)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  153.898,1470.24 180.284,1470.24 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4000)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  153.898,1166.68 180.284,1166.68 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4000)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  153.898,863.131 180.284,863.131 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4000)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  153.898,559.577 180.284,559.577 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4000)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  153.898,256.023 180.284,256.023 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip4000)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 209.192, 1541.47)\" x=\"209.192\" y=\"1541.47\">0</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip4000)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 902.969, 1541.47)\" x=\"902.969\" y=\"1541.47\">100</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip4000)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 1596.75, 1541.47)\" x=\"1596.75\" y=\"1541.47\">200</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip4000)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 2290.52, 1541.47)\" x=\"2290.52\" y=\"1541.47\">300</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip4000)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 129.898, 1487.74)\" x=\"129.898\" y=\"1487.74\">0.0</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip4000)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 129.898, 1184.18)\" x=\"129.898\" y=\"1184.18\">0.2</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip4000)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 129.898, 880.631)\" x=\"129.898\" y=\"880.631\">0.4</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip4000)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 129.898, 577.077)\" x=\"129.898\" y=\"577.077\">0.6</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip4000)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 129.898, 273.523)\" x=\"129.898\" y=\"273.523\">0.8</text>\n",
       "</g>\n",
       "<polyline clip-path=\"url(#clip4002)\" style=\"stroke:#009af9; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  216.13,94.2282 223.068,88.0053 230.005,362.418 236.943,612.547 243.881,790.581 250.819,910.485 257.756,995.329 264.694,1044.66 271.632,1078.96 278.57,1110.83 \n",
       "  285.508,1142.7 292.445,1168.81 299.383,1181.41 306.321,1203.87 313.259,1215.1 320.196,1226.64 327.134,1239.99 334.072,1252.59 341.01,1264.13 347.948,1269.13 \n",
       "  354.885,1274.14 361.823,1286.74 368.761,1295.85 375.699,1301.31 382.636,1310.72 389.574,1317.1 396.512,1323.32 403.45,1324.99 410.387,1329.09 417.325,1332.73 \n",
       "  424.263,1336.83 431.201,1342.44 438.139,1340.32 445.076,1344.72 452.014,1351.55 458.952,1353.52 465.89,1360.2 472.827,1366.58 479.765,1368.24 486.703,1368.09 \n",
       "  493.641,1376.14 500.579,1374.62 507.516,1376.59 514.454,1383.12 521.392,1383.88 528.33,1384.64 535.267,1387.22 542.205,1389.8 549.143,1391.01 556.081,1389.95 \n",
       "  563.018,1392.23 569.956,1395.41 576.894,1396.63 583.832,1399.97 590.77,1403 597.707,1405.28 604.645,1404.82 611.583,1407.1 618.521,1408.77 625.458,1410.44 \n",
       "  632.396,1404.37 639.334,1407.86 646.272,1413.17 653.21,1412.11 660.147,1414.99 667.085,1415.14 674.023,1418.48 680.961,1417.88 687.898,1418.63 694.836,1419.85 \n",
       "  701.774,1416.36 708.712,1420.76 715.649,1418.03 722.587,1422.28 729.525,1421.37 736.463,1423.34 743.401,1425.77 750.338,1425.16 757.276,1422.73 764.214,1424.86 \n",
       "  771.152,1423.64 778.089,1422.58 785.027,1423.34 791.965,1424.1 798.903,1426.53 805.841,1427.59 812.778,1427.13 819.716,1431.08 826.654,1427.44 833.592,1431.54 \n",
       "  840.529,1429.71 847.467,1429.26 854.405,1429.56 861.343,1430.63 868.281,1429.26 875.218,1428.35 882.156,1432.75 889.094,1435.03 896.032,1433.81 902.969,1431.23 \n",
       "  909.907,1432.75 916.845,1435.94 923.783,1432.75 930.72,1433.96 937.658,1433.96 944.596,1436.85 951.534,1435.03 958.472,1436.39 965.409,1435.79 972.347,1435.48 \n",
       "  979.285,1435.18 986.223,1434.42 993.16,1434.42 1000.1,1436.85 1007.04,1434.87 1013.97,1435.33 1020.91,1437.15 1027.85,1437.61 1034.79,1436.24 1041.72,1438.21 \n",
       "  1048.66,1435.03 1055.6,1438.82 1062.54,1436.54 1069.48,1439.12 1076.41,1438.82 1083.35,1439.28 1090.29,1436.85 1097.23,1439.12 1104.16,1439.28 1111.1,1438.82 \n",
       "  1118.04,1440.49 1124.98,1440.04 1131.92,1439.88 1138.85,1438.37 1145.79,1441.7 1152.73,1437.61 1159.67,1442.92 1166.6,1440.19 1173.54,1442.62 1180.48,1440.64 \n",
       "  1187.42,1443.22 1194.36,1442.77 1201.29,1442.01 1208.23,1439.73 1215.17,1442.46 1222.11,1443.37 1229.04,1441.86 1235.98,1441.4 1242.92,1444.44 1249.86,1444.13 \n",
       "  1256.8,1442.46 1263.73,1442.77 1270.67,1440.79 1277.61,1439.73 1284.55,1440.79 1291.48,1442.01 1298.42,1441.55 1305.36,1442.92 1312.3,1442.31 1319.24,1443.37 \n",
       "  1326.17,1441.25 1333.11,1443.22 1340.05,1441.7 1346.99,1439.43 1353.92,1441.4 1360.86,1442.62 1367.8,1443.83 1374.74,1439.43 1381.68,1440.79 1388.61,1442.92 \n",
       "  1395.55,1442.46 1402.49,1443.22 1409.43,1441.55 1416.36,1439.73 1423.3,1442.92 1430.24,1438.67 1437.18,1445.65 1444.12,1445.65 1451.05,1443.22 1457.99,1440.19 \n",
       "  1464.93,1445.04 1471.87,1443.98 1478.8,1439.28 1485.74,1440.49 1492.68,1443.53 1499.62,1444.13 1506.56,1443.22 1513.49,1442.77 1520.43,1442.92 1527.37,1442.31 \n",
       "  1534.31,1443.68 1541.24,1446.71 1548.18,1443.37 1555.12,1444.74 1562.06,1445.04 1569,1444.59 1575.93,1441.86 1582.87,1442.01 1589.81,1445.5 1596.75,1442.46 \n",
       "  1603.68,1443.37 1610.62,1441.4 1617.56,1442.46 1624.5,1441.7 1631.44,1445.35 1638.37,1444.44 1645.31,1440.19 1652.25,1443.37 1659.19,1444.29 1666.12,1440.64 \n",
       "  1673.06,1438.06 1680,1439.43 1686.94,1439.43 1693.88,1441.1 1700.81,1439.43 1707.75,1439.58 1714.69,1442.46 1721.63,1439.58 1728.56,1440.04 1735.5,1443.22 \n",
       "  1742.44,1442.16 1749.38,1440.64 1756.32,1438.82 1763.25,1440.49 1770.19,1439.58 1777.13,1440.79 1784.07,1440.49 1791,1442.31 1797.94,1441.86 1804.88,1438.82 \n",
       "  1811.82,1440.95 1818.76,1438.21 1825.69,1442.16 1832.63,1442.92 1839.57,1439.88 1846.51,1439.28 1853.44,1443.68 1860.38,1442.01 1867.32,1440.19 1874.26,1443.22 \n",
       "  1881.2,1444.89 1888.13,1444.13 1895.07,1444.44 1902.01,1441.25 1908.95,1439.12 1915.88,1441.55 1922.82,1442.16 1929.76,1441.1 1936.7,1441.7 1943.64,1441.7 \n",
       "  1950.57,1442.16 1957.51,1443.37 1964.45,1444.74 1971.39,1444.89 1978.32,1442.46 1985.26,1443.07 1992.2,1441.1 1999.14,1440.95 2006.08,1443.98 2013.01,1443.53 \n",
       "  2019.95,1441.55 2026.89,1443.07 2033.83,1443.07 2040.76,1443.68 2047.7,1440.64 2054.64,1443.22 2061.58,1443.07 2068.52,1443.07 2075.45,1443.07 2082.39,1440.34 \n",
       "  2089.33,1443.53 2096.27,1442.16 2103.2,1441.4 2110.14,1442.46 2117.08,1445.04 2124.02,1442.62 2130.96,1442.62 2137.89,1441.7 2144.83,1444.13 2151.77,1443.22 \n",
       "  2158.71,1441.86 2165.64,1441.86 2172.58,1444.59 2179.52,1444.89 2186.46,1446.26 2193.4,1445.8 2200.33,1443.22 2207.27,1446.56 2214.21,1444.13 2221.15,1446.26 \n",
       "  2228.08,1445.8 2235.02,1445.04 2241.96,1443.98 2248.9,1445.8 2255.84,1444.74 2262.77,1444.74 2269.71,1445.35 2276.65,1445.65 2283.59,1444.89 2290.52,1444.29 \n",
       "  \n",
       "  \"/>\n",
       "<path clip-path=\"url(#clip4000)\" d=\"\n",
       "M1856.12 251.724 L2280.76 251.724 L2280.76 130.764 L1856.12 130.764  Z\n",
       "  \" fill=\"#ffffff\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<polyline clip-path=\"url(#clip4000)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1856.12,251.724 2280.76,251.724 2280.76,130.764 1856.12,130.764 1856.12,251.724 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4000)\" style=\"stroke:#009af9; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1880.12,191.244 2024.12,191.244 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip4000)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:start;\" transform=\"rotate(0, 2048.12, 208.744)\" x=\"2048.12\" y=\"208.744\">double q</text>\n",
       "</g>\n",
       "</svg>\n"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stats = []\n",
    "for _ in 1:10000\n",
    "    hook = CountOfLeft()\n",
    "    run(init_double_Q_agent(), env, StopAfterEpisode(300),hook)\n",
    "    push!(stats, hook.counts)\n",
    "end\n",
    "\n",
    "plot(mean(stats), legend=:topright, label=\"double q\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32mProgress: 100%|█████████████████████████████████████████| Time: 0:00:00\u001b[39m\n",
      "\u001b[32mProgress: 100%|█████████████████████████████████████████| Time: 0:00:00\u001b[39m\n"
     ]
    },
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" width=\"600\" height=\"400\" viewBox=\"0 0 2400 1600\">\n",
       "<defs>\n",
       "  <clipPath id=\"clip4400\">\n",
       "    <rect x=\"0\" y=\"0\" width=\"2400\" height=\"1600\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<path clip-path=\"url(#clip4400)\" d=\"\n",
       "M0 1600 L2400 1600 L2400 0 L0 0  Z\n",
       "  \" fill=\"#ffffff\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<defs>\n",
       "  <clipPath id=\"clip4401\">\n",
       "    <rect x=\"480\" y=\"0\" width=\"1681\" height=\"1600\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<path clip-path=\"url(#clip4400)\" d=\"\n",
       "M153.898 1487.47 L2352.76 1487.47 L2352.76 47.2441 L153.898 47.2441  Z\n",
       "  \" fill=\"#ffffff\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<defs>\n",
       "  <clipPath id=\"clip4402\">\n",
       "    <rect x=\"153\" y=\"47\" width=\"2200\" height=\"1441\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<polyline clip-path=\"url(#clip4402)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  209.192,1487.47 209.192,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4402)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  902.969,1487.47 902.969,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4402)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  1596.75,1487.47 1596.75,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4402)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  2290.52,1487.47 2290.52,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4402)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  153.898,1470.13 2352.76,1470.13 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4402)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  153.898,1168.02 2352.76,1168.02 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4402)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  153.898,865.921 2352.76,865.921 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4402)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  153.898,563.818 2352.76,563.818 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4402)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  153.898,261.715 2352.76,261.715 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4400)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  153.898,1487.47 2352.76,1487.47 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4400)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  153.898,1487.47 153.898,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4400)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  209.192,1487.47 209.192,1470.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4400)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  902.969,1487.47 902.969,1470.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4400)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1596.75,1487.47 1596.75,1470.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4400)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  2290.52,1487.47 2290.52,1470.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4400)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  153.898,1470.13 180.284,1470.13 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4400)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  153.898,1168.02 180.284,1168.02 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4400)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  153.898,865.921 180.284,865.921 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4400)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  153.898,563.818 180.284,563.818 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4400)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  153.898,261.715 180.284,261.715 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip4400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 209.192, 1541.47)\" x=\"209.192\" y=\"1541.47\">0</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip4400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 902.969, 1541.47)\" x=\"902.969\" y=\"1541.47\">100</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip4400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 1596.75, 1541.47)\" x=\"1596.75\" y=\"1541.47\">200</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip4400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 2290.52, 1541.47)\" x=\"2290.52\" y=\"1541.47\">300</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip4400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 129.898, 1487.63)\" x=\"129.898\" y=\"1487.63\">0.0</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip4400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 129.898, 1185.52)\" x=\"129.898\" y=\"1185.52\">0.2</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip4400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 129.898, 883.421)\" x=\"129.898\" y=\"883.421\">0.4</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip4400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 129.898, 581.318)\" x=\"129.898\" y=\"581.318\">0.6</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip4400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 129.898, 279.215)\" x=\"129.898\" y=\"279.215\">0.8</text>\n",
       "</g>\n",
       "<polyline clip-path=\"url(#clip4402)\" style=\"stroke:#009af9; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  216.13,100.694 223.068,94.5006 230.005,367.602 236.943,616.534 243.881,793.718 250.819,913.049 257.756,997.486 264.694,1046.58 271.632,1080.72 278.57,1112.44 \n",
       "  285.508,1144.16 292.445,1170.14 299.383,1182.68 306.321,1205.03 313.259,1216.21 320.196,1227.69 327.134,1240.98 334.072,1253.52 341.01,1265 347.948,1269.98 \n",
       "  354.885,1274.97 361.823,1287.51 368.761,1296.57 375.699,1302.01 382.636,1311.37 389.574,1317.72 396.512,1323.91 403.45,1325.57 410.387,1329.65 417.325,1333.27 \n",
       "  424.263,1337.35 431.201,1342.94 438.139,1340.83 445.076,1345.21 452.014,1352 458.952,1353.97 465.89,1360.61 472.827,1366.96 479.765,1368.62 486.703,1368.47 \n",
       "  493.641,1376.47 500.579,1374.96 507.516,1376.93 514.454,1383.42 521.392,1384.18 528.33,1384.93 535.267,1387.5 542.205,1390.07 549.143,1391.28 556.081,1390.22 \n",
       "  563.018,1392.49 569.956,1395.66 576.894,1396.87 583.832,1400.19 590.77,1403.21 597.707,1405.48 604.645,1405.02 611.583,1407.29 618.521,1408.95 625.458,1410.61 \n",
       "  632.396,1404.57 639.334,1408.04 646.272,1413.33 653.21,1412.27 660.147,1415.14 667.085,1415.29 674.023,1418.62 680.961,1418.01 687.898,1418.77 694.836,1419.98 \n",
       "  701.774,1416.5 708.712,1420.88 715.649,1418.16 722.587,1422.39 729.525,1421.49 736.463,1423.45 743.401,1425.87 750.338,1425.26 757.276,1422.85 764.214,1424.96 \n",
       "  771.152,1423.75 778.089,1422.7 785.027,1423.45 791.965,1424.21 798.903,1426.62 805.841,1427.68 812.778,1427.23 819.716,1431.16 826.654,1427.53 833.592,1431.61 \n",
       "  840.529,1429.8 847.467,1429.34 854.405,1429.64 861.343,1430.7 868.281,1429.34 875.218,1428.44 882.156,1432.82 889.094,1435.08 896.032,1433.87 902.969,1431.31 \n",
       "  909.907,1432.82 916.845,1435.99 923.783,1432.82 930.72,1434.03 937.658,1434.03 944.596,1436.9 951.534,1435.08 958.472,1436.44 965.409,1435.84 972.347,1435.54 \n",
       "  979.285,1435.23 986.223,1434.48 993.16,1434.48 1000.1,1436.9 1007.04,1434.93 1013.97,1435.38 1020.91,1437.2 1027.85,1437.65 1034.79,1436.29 1041.72,1438.25 \n",
       "  1048.66,1435.08 1055.6,1438.86 1062.54,1436.59 1069.48,1439.16 1076.41,1438.86 1083.35,1439.31 1090.29,1436.9 1097.23,1439.16 1104.16,1439.31 1111.1,1438.86 \n",
       "  1118.04,1440.52 1124.98,1440.07 1131.92,1439.92 1138.85,1438.41 1145.79,1441.73 1152.73,1437.65 1159.67,1442.94 1166.6,1440.22 1173.54,1442.64 1180.48,1440.67 \n",
       "  1187.42,1443.24 1194.36,1442.79 1201.29,1442.03 1208.23,1439.77 1215.17,1442.48 1222.11,1443.39 1229.04,1441.88 1235.98,1441.43 1242.92,1444.45 1249.86,1444.15 \n",
       "  1256.8,1442.48 1263.73,1442.79 1270.67,1440.82 1277.61,1439.77 1284.55,1440.82 1291.48,1442.03 1298.42,1441.58 1305.36,1442.94 1312.3,1442.33 1319.24,1443.39 \n",
       "  1326.17,1441.28 1333.11,1443.24 1340.05,1441.73 1346.99,1439.46 1353.92,1441.43 1360.86,1442.64 1367.8,1443.84 1374.74,1439.46 1381.68,1440.82 1388.61,1442.94 \n",
       "  1395.55,1442.48 1402.49,1443.24 1409.43,1441.58 1416.36,1439.77 1423.3,1442.94 1430.24,1438.71 1437.18,1445.66 1444.12,1445.66 1451.05,1443.24 1457.99,1440.22 \n",
       "  1464.93,1445.05 1471.87,1443.99 1478.8,1439.31 1485.74,1440.52 1492.68,1443.54 1499.62,1444.15 1506.56,1443.24 1513.49,1442.79 1520.43,1442.94 1527.37,1442.33 \n",
       "  1534.31,1443.69 1541.24,1446.71 1548.18,1443.39 1555.12,1444.75 1562.06,1445.05 1569,1444.6 1575.93,1441.88 1582.87,1442.03 1589.81,1445.51 1596.75,1442.48 \n",
       "  1603.68,1443.39 1610.62,1441.43 1617.56,1442.48 1624.5,1441.73 1631.44,1445.35 1638.37,1444.45 1645.31,1440.22 1652.25,1443.39 1659.19,1444.3 1666.12,1440.67 \n",
       "  1673.06,1438.1 1680,1439.46 1686.94,1439.46 1693.88,1441.12 1700.81,1439.46 1707.75,1439.61 1714.69,1442.48 1721.63,1439.61 1728.56,1440.07 1735.5,1443.24 \n",
       "  1742.44,1442.18 1749.38,1440.67 1756.32,1438.86 1763.25,1440.52 1770.19,1439.61 1777.13,1440.82 1784.07,1440.52 1791,1442.33 1797.94,1441.88 1804.88,1438.86 \n",
       "  1811.82,1440.97 1818.76,1438.25 1825.69,1442.18 1832.63,1442.94 1839.57,1439.92 1846.51,1439.31 1853.44,1443.69 1860.38,1442.03 1867.32,1440.22 1874.26,1443.24 \n",
       "  1881.2,1444.9 1888.13,1444.15 1895.07,1444.45 1902.01,1441.28 1908.95,1439.16 1915.88,1441.58 1922.82,1442.18 1929.76,1441.12 1936.7,1441.73 1943.64,1441.73 \n",
       "  1950.57,1442.18 1957.51,1443.39 1964.45,1444.75 1971.39,1444.9 1978.32,1442.48 1985.26,1443.09 1992.2,1441.12 1999.14,1440.97 2006.08,1443.99 2013.01,1443.54 \n",
       "  2019.95,1441.58 2026.89,1443.09 2033.83,1443.09 2040.76,1443.69 2047.7,1440.67 2054.64,1443.24 2061.58,1443.09 2068.52,1443.09 2075.45,1443.09 2082.39,1440.37 \n",
       "  2089.33,1443.54 2096.27,1442.18 2103.2,1441.43 2110.14,1442.48 2117.08,1445.05 2124.02,1442.64 2130.96,1442.64 2137.89,1441.73 2144.83,1444.15 2151.77,1443.24 \n",
       "  2158.71,1441.88 2165.64,1441.88 2172.58,1444.6 2179.52,1444.9 2186.46,1446.26 2193.4,1445.81 2200.33,1443.24 2207.27,1446.56 2214.21,1444.15 2221.15,1446.26 \n",
       "  2228.08,1445.81 2235.02,1445.05 2241.96,1443.99 2248.9,1445.81 2255.84,1444.75 2262.77,1444.75 2269.71,1445.35 2276.65,1445.66 2283.59,1444.9 2290.52,1444.3 \n",
       "  \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4402)\" style=\"stroke:#e26f46; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  216.13,91.6306 223.068,102.355 230.005,96.7663 236.943,93.4432 243.881,98.73 250.819,91.4795 257.756,90.1201 264.694,88.0053 271.632,96.4642 278.57,100.543 \n",
       "  285.508,97.0684 292.445,102.204 299.383,97.2195 306.321,104.319 313.259,95.709 320.196,100.392 327.134,99.7874 334.072,104.47 341.01,109.908 347.948,111.116 \n",
       "  354.885,118.518 361.823,123.653 368.761,121.237 375.699,121.69 382.636,125.919 389.574,135.889 396.512,139.665 403.45,133.623 410.387,150.541 417.325,155.072 \n",
       "  424.263,151.145 431.201,165.797 438.139,167.156 445.076,167.156 452.014,173.047 458.952,176.975 465.89,180.751 472.827,189.663 479.765,202.653 486.703,212.17 \n",
       "  493.641,216.701 500.579,221.384 507.516,231.353 514.454,244.193 521.392,247.214 528.33,251.745 535.267,266.397 542.205,276.367 549.143,283.919 556.081,299.78 \n",
       "  563.018,304.613 569.956,319.416 576.894,319.416 583.832,330.443 590.77,337.996 597.707,354.007 604.645,369.867 611.583,385.124 618.521,390.561 625.458,399.171 \n",
       "  632.396,408.083 639.334,420.772 646.272,432.403 653.21,446.451 660.147,453.399 667.085,468.202 674.023,483.911 680.961,492.219 687.898,503.548 694.836,514.575 \n",
       "  701.774,525.753 708.712,548.41 715.649,555.661 722.587,569.255 729.525,578.621 736.463,590.1 743.401,608.68 750.338,613.362 757.276,624.389 764.214,641.458 \n",
       "  771.152,653.542 778.089,663.663 785.027,677.408 791.965,685.867 798.903,702.483 805.841,715.624 812.778,732.24 819.716,740.246 826.654,753.236 833.592,764.716 \n",
       "  840.529,771.06 847.467,786.77 854.405,797.192 861.343,816.074 868.281,824.079 875.218,832.689 882.156,843.565 889.094,854.138 896.032,861.238 902.969,871.509 \n",
       "  909.907,885.557 916.845,890.693 923.783,903.683 930.72,916.976 937.658,928.607 944.596,938.425 951.534,945.978 958.472,954.135 965.409,962.593 972.347,974.527 \n",
       "  979.285,976.49 986.223,993.559 993.16,1000.05 1000.1,1006.55 1007.04,1015.16 1013.97,1021.5 1020.91,1034.49 1027.85,1036.76 1034.79,1050.35 1041.72,1054.13 \n",
       "  1048.66,1064.55 1055.6,1071.95 1062.54,1079.96 1069.48,1082.98 1076.41,1094.91 1083.35,1102.01 1090.29,1106.7 1097.23,1114.25 1104.16,1119.84 1111.1,1122.71 \n",
       "  1118.04,1130.41 1124.98,1139.47 1131.92,1143.86 1138.85,1153.22 1145.79,1154.73 1152.73,1165.15 1159.67,1166.06 1166.6,1176.03 1173.54,1182.22 1180.48,1191.29 \n",
       "  1187.42,1191.29 1194.36,1198.38 1201.29,1204.58 1208.23,1212.89 1215.17,1219.38 1222.11,1219.08 1229.04,1220.44 1235.98,1227.99 1242.92,1231.62 1249.86,1234.79 \n",
       "  1256.8,1239.77 1263.73,1245.06 1270.67,1248.08 1277.61,1250.5 1284.55,1252.16 1291.48,1261.07 1298.42,1262.73 1305.36,1263.94 1312.3,1270.13 1319.24,1273.46 \n",
       "  1326.17,1276.03 1333.11,1279.95 1340.05,1282.67 1346.99,1282.97 1353.92,1289.47 1360.86,1291.13 1367.8,1294.6 1374.74,1297.93 1381.68,1304.57 1388.61,1303.06 \n",
       "  1395.55,1308.35 1402.49,1313.94 1409.43,1312.13 1416.36,1317.11 1423.3,1317.72 1430.24,1319.38 1437.18,1322.55 1444.12,1327.99 1451.05,1328.74 1457.99,1327.53 \n",
       "  1464.93,1332.67 1471.87,1332.82 1478.8,1337.5 1485.74,1340.37 1492.68,1344 1499.62,1346.87 1506.56,1348.23 1513.49,1349.59 1520.43,1352.91 1527.37,1353.06 \n",
       "  1534.31,1355.03 1541.24,1358.05 1548.18,1356.54 1555.12,1359.25 1562.06,1361.37 1569,1361.97 1575.93,1365.75 1582.87,1367.41 1589.81,1367.41 1596.75,1369.98 \n",
       "  1603.68,1372.4 1610.62,1371.79 1617.56,1377.08 1624.5,1379.34 1631.44,1377.23 1638.37,1380.1 1645.31,1382.67 1652.25,1380.7 1659.19,1381.01 1666.12,1381.91 \n",
       "  1673.06,1384.93 1680,1381.61 1686.94,1385.08 1693.88,1390.22 1700.81,1391.28 1707.75,1389.16 1714.69,1390.67 1721.63,1388.11 1728.56,1388.41 1735.5,1392.49 \n",
       "  1742.44,1391.43 1749.38,1394.9 1756.32,1391.13 1763.25,1392.33 1770.19,1392.64 1777.13,1396.11 1784.07,1395.96 1791,1395.05 1797.94,1396.87 1804.88,1397.92 \n",
       "  1811.82,1400.49 1818.76,1400.94 1825.69,1398.53 1832.63,1400.64 1839.57,1399.89 1846.51,1400.34 1853.44,1399.13 1860.38,1401.25 1867.32,1404.12 1874.26,1406.84 \n",
       "  1881.2,1406.84 1888.13,1405.78 1895.07,1409.4 1902.01,1405.93 1908.95,1409.4 1915.88,1410.61 1922.82,1411.82 1929.76,1410.76 1936.7,1413.03 1943.64,1410.61 \n",
       "  1950.57,1413.18 1957.51,1413.63 1964.45,1413.63 1971.39,1412.42 1978.32,1414.24 1985.26,1416.65 1992.2,1412.88 1999.14,1414.99 2006.08,1413.03 2013.01,1414.99 \n",
       "  2019.95,1418.01 2026.89,1415.29 2033.83,1414.84 2040.76,1417.56 2047.7,1417.56 2054.64,1418.16 2061.58,1418.01 2068.52,1415.29 2075.45,1420.43 2082.39,1419.22 \n",
       "  2089.33,1421.19 2096.27,1417.56 2103.2,1417.11 2110.14,1419.83 2117.08,1419.52 2124.02,1423 2130.96,1419.22 2137.89,1420.58 2144.83,1419.37 2151.77,1420.13 \n",
       "  2158.71,1421.94 2165.64,1419.68 2172.58,1424.51 2179.52,1423.45 2186.46,1423 2193.4,1422.55 2200.33,1423 2207.27,1424.96 2214.21,1423.9 2221.15,1427.98 \n",
       "  2228.08,1422.85 2235.02,1426.77 2241.96,1425.26 2248.9,1425.26 2255.84,1426.17 2262.77,1423.45 2269.71,1425.72 2276.65,1426.17 2283.59,1427.68 2290.52,1429.49 \n",
       "  \n",
       "  \"/>\n",
       "<path clip-path=\"url(#clip4400)\" d=\"\n",
       "M1856.12 312.204 L2280.76 312.204 L2280.76 130.764 L1856.12 130.764  Z\n",
       "  \" fill=\"#ffffff\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<polyline clip-path=\"url(#clip4400)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1856.12,312.204 2280.76,312.204 2280.76,130.764 1856.12,130.764 1856.12,312.204 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4400)\" style=\"stroke:#009af9; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1880.12,191.244 2024.12,191.244 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip4400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:start;\" transform=\"rotate(0, 2048.12, 208.744)\" x=\"2048.12\" y=\"208.744\">double q</text>\n",
       "</g>\n",
       "<polyline clip-path=\"url(#clip4400)\" style=\"stroke:#e26f46; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1880.12,251.724 2024.12,251.724 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip4400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:start;\" transform=\"rotate(0, 2048.12, 269.224)\" x=\"2048.12\" y=\"269.224\">q</text>\n",
       "</g>\n",
       "</svg>\n"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stats = []\n",
    "for _ in 1:10000\n",
    "    hook = CountOfLeft()\n",
    "    run(init_Q_agent(), env, StopAfterEpisode(300),hook)\n",
    "    push!(stats, hook.counts)\n",
    "end\n",
    "plot!(mean(stats), legend=:topright, label=\"q\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TODO\n",
    "\n",
    "We need to add a legal action here to restrict that the actions of the first step can only be 1(left) or 2(right). So that the figure will look exactly like the one on the book."
   ]
  }
 ],
 "metadata": {
  "@webio": {
   "lastCommId": null,
   "lastKernelId": null
  },
  "kernelspec": {
   "display_name": "Julia 1.3.0",
   "language": "julia",
   "name": "julia-1.3"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "1.3.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
